package com.bigdata.collector.labels;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bigdata.collector.model.Page;
import com.bigdata.collector.util.HttpUtils;

public class Collector4Dict591hx extends AbstractCollector {

	private static Logger logger = Logger.getLogger(Collector4Dict591hx.class);
	private String category;

	public Collector4Dict591hx(String seed) {
		super(seed);

	}

	public Collector4Dict591hx(String seed, String category) {
		this(seed);
		this.category = category;
	}

	public static void main(String[] args) {
		String[] seeds = { "GuPiao", "JiJin", "QiHuo", "WaiHui", "ZhaiQuan",
				"BaoXian", "YinXing", "LiCai", "FangChan", "JingJi" };
		int i = 1;
		for (String seed : seeds) {

			Collector4Dict591hx ct = new Collector4Dict591hx(
					"http://dict.591hx.com/liebiao/" + seed, seed);
			i++;
			ct.run();
		}
		// save("万福生科（湖南）农业开发股份有限公司", 2 + "");
	}

	@Override
	public void crawler(String url) {
		try {
			int pageNum = 1;
			String previous = null;
			while (true) {
				String subUrl = url + "_" + pageNum + ".shtml";

				Page page = HttpUtils.fetchHttpResponse(subUrl);
				if (page == null)
					break;
				Document doc = Jsoup.parse(new String(page.content, "utf-8"));
				if (previous == doc.text())
					break;
				previous = doc.text();
				if (!extractTags(doc))
					break;
				pageNum++;
			}
		} catch (Exception e) {
			// logger.error(ExceptionUtils.getStackTrace(e));
		}
	}

	@Override
	boolean extractTags(Document doc) {
		if (doc.getElementById("lb_lb") == null)
			return false;
		String category = doc.getElementById("lb_r1_c1_r3")
				.getElementsByClass("lb_bg").get(0).getElementsByTag("h2")
				.text();
		if (category == null || category.equals(""))
			category = this.category;
		Elements es = doc.getElementById("lb_lb").getElementsByTag("a");
		for (Element e : es) {
			String label = e.text().trim();
			super.save2File("dict591hx", label, category, true);
		}
		return true;
	}

	@Override
	public void extractURLs(Document doc) {

	}
}
